{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.59076524,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('MountainCar-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "\n",
    "        if done:\n",
    "            reward = 100.0\n",
    "\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhgklEQVR4nO3da2xT9/0/8Pex42sS2yEXm9waWi4hXAK5mwRaICVlrC3QbV3VB1U3taINU2mrSs2DUk2alGqV/tq6VfTBtrYPtjExla1FNG1IIBfikAukhACh6aAJJE5Cgu1cHMf2+fwftDk/Uijkfmzn85KORM752v74mPPWOd9zzvcIRERgjDEZKOQugDG2eHEAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkwwHEGJMNBxBjTDYcQIwx2XAAMcZkI1sAvf/++0hJSYFWq0Vubi4aGhrkKoUxJhNZAuhf//oXXnvtNbz99ts4e/Ys0tPTUVRUhL6+PjnKYYzJRJDjZtTc3FxkZ2fjz3/+MwBAFEUkJSXhN7/5Dd58882FLocxJpOwhf7A8fFxNDc3o6SkRJqnUChQWFgIm81219d4PB54PB7pb1EUMTg4iOjoaAiCMO81M8amh4gwNDSE+Ph4KBQ/fqC14AF08+ZN+P1+mM3mSfPNZjMuX75819eUlpbit7/97UKUxxibQ11dXUhMTPzR5QseQDNRUlKC1157Tfrb6XQiOTkZXV1dMBgMMlbGGLsbl8uFpKQkREZG3rPdggdQTEwMlEolent7J83v7e2FxWK562s0Gg00Gs0d8w0GAwcQYwHsfl0kC34WTK1WIzMzExUVFdI8URRRUVEBq9W60OUwxmQkyyHYa6+9hueeew5ZWVnIycnBH/7wB4yMjOD555+XoxzGmExkCaCnn34a/f39OHjwIOx2OzZs2ICysrI7OqYZY6FNluuAZsvlcsFoNMLpdHIfEGPzzOu1Y3j4NLTaVKhUS6FQREKhUN3zNVPdRoPiLBhjTD4ez1VcvfosBEEDlWoptNqV0OnWQ6/PgEazHGp1MpTKcABh074ujwOIMXZfRF4QeeDxuODxtMPp/AyAAIXCAJXKDL0+C0bjT7BkydMQhKnHCgcQY2walFAotAgLs0CrXQ6dbt33e0IrodEsA6Cc1rtxADHG7snjEaBWb0ZUVA50uvXQ6dZCpUqAUmmEQqGe1XtzADHG7unqVSVUqv+HhIQNAIQ5vf+SA4gxdk/fnSdXQhDm/rplHhGRMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuF7wRhj90VEmBg8lYgm3ZA6m5tTOYAYY5MQEVwuF4aHh9HT0wObzQav14uWlhbodDoMDw9Dp9NBpVJBq9ViyZIliIiIQEJCAiIjI6FWT32IDg4gxhY5IoLX60V3dzfa2trgcDjgdrthsVgQGxuLgoICREdHw2QyTXrMsiiKGBsbw+DgIHp6enD16lU4nU6Eh4cjKipqSp/NAcTYIkREEEURPT09OHfuHG7evImkpCSkpaVh6dKlkx4Eeq9DLKPRCLPZjNTUVGmew+FAe3v7lOrgAGJskfF4PLhy5QqamppgsViQkZGBuLg4hIVNf1D5Cbe/LioqCmlpaVN6HQcQY4vAxGFWa2srLly4gNTUVPz85z9HRESErHVxADEW4vx+P9ra2tDa2orVq1fjmWeegUqlmtOhVWeKA4ixEEVEuHHjBhobGxETE4Pdu3dDr9cHRPBM4ABiLASNj4+joaEBvb292LZtG0wmU0AFzwQOIMZCCBGhu7sbJ0+eRFZWFqxWK5TK6T2rayFxADEWIvx+P1paWtDb24snnngCkZGRAbnXc7tp3wtWXV2Nxx9/HPHx8RAEAf/5z38mLSciHDx4EEuXLoVOp0NhYSG+/vrrSW0GBwfx7LPPwmAwwGQy4de//jWGh4dn9UUYW8xGRkbwxRdfQBAEFBUVwWAwBHz4ADMIoJGREaSnp+P999+/6/Lf//73eO+99/DBBx/gzJkzCA8PR1FREcbGxqQ2zz77LNra2lBeXo5jx46huroaL7744sy/BWOLFBGhq6sLJ06cwMaNG7Fx48aAPuS6A80CADp69Kj0tyiKZLFY6N1335XmORwO0mg09M9//pOIiC5evEgAqLGxUWrz+eefkyAIdOPGjSl9rtPpJADkdDpnUz5jQU0URbp8+TL9+9//puHhYRJFUe6SJFPdRud0OI6rV6/CbrejsLBQmmc0GpGbmwubzQYAsNlsMJlMyMrKktoUFhZCoVDgzJkzd31fj8cDl8s1aWJsMfP5fLDZbOjv78cTTzyB8PDwoDjk+qE5DSC73Q4AMJvNk+abzWZpmd1uR1xc3KTlYWFhWLJkidTmh0pLS2E0GqUpKSlpLstmLKh4vV7U1NRApVIhPz8fKpVK7pJmLCgGJCspKYHT6ZSmrq4uuUtibMEREcbGxlBVVYXk5GRkZWUF5V7P7eY0gCwWCwCgt7d30vze3l5pmcViQV9f36TlPp8Pg4ODUpsf0mg0MBgMkybGFhMigsfjQXl5OVatWoUHH3ww6MMHmOMAWrZsGSwWCyoqKqR5LpcLZ86cgdVqBQBYrVY4HA40NzdLbSorKyGKInJzc+eyHMZCxsjICCorK5GXl4fExMSQCB9gBhciDg8Po6OjQ/r76tWraGlpwZIlS5CcnIwDBw7gd7/7HVasWIFly5bhrbfeQnx8PHbv3g0AWL16NR577DG88MIL+OCDD+D1erF//3788pe/RHx8/Jx9McZCARFheHgYFRUV2LJlC5YsWSJ3SXNruqfXTp48SQDumJ577jki+u7U4FtvvUVms5k0Gg1t376d2tvbJ73HwMAAPfPMMxQREUEGg4Gef/55GhoamnINfBqeLRYul4s+//xzunXrVkCdZr+fqW6jAtH3I00HEZfLBaPRCKfTyf1BLCQRERwOB06fPo2CggKYTCa5S5qWqW6jQXEWjLHFhIgwOjqK06dPw2q1wmg0yl3SvOGbURkLMGNjYzh16hQ2bdoUen0+P8B7QIwFEI/Hg4qKCmRlZU35yRLBjAOIsQAxcYXzhg0bEBcXFzKn2u+FA4ixAOD1elFRUYEHH3wQCQkJiyJ8AA4gxmRHRGhoaIDFYsGyZcsWTfgAHECMyYqI0NraCrVajfT09EUVPgAHEGOyISJcuHAB3d3dyMzMXHThA3AAMSabnp4edHd3Y9u2bZOeub6YLM5vzZjMJm7I3rJlC9RqtdzlyIYDiLEF5na7cfz4cVitVmi1WrnLkVVQXwkdhLexsUXO7/ejuroaDz/8MGJiYuQuR3ZBvQfkcrk4hFjQEEURNpsNiYmJPPTM94I6gOrq6jA+Pi53GYzdFxHh0qVLGB8fR1pa2qI843U3QR1AqampqKiogM/nk7sUxn4UEaG3txddXV3YvHkzh89tgjqAUlJSEBMTg5aWFj4UYwFreHgYNpsNW7ZsCeonWMyHoA4gQRCQmZkJl8uFb7/9lkOIBRyv14tTp04hOzsbOp1O7nICTlAHEAAolUoUFBSgqakJt27dkrscxiSiKKK+vh4rV65cVDeYTkfQBxAAqFQqPPLII6ipqYHH45G7HMak2ywUCgVWrFjB4fMjQiKABEFAdHQ01q1bh7q6OoiiKHdJbBEjIvT39+Pq1avIzc1dtLdZTEXIrBlBELBs2TKEh4fj3Llz3B/EZON2u1FfX49t27YhLCyor/WddyETQMB3IZSRkQG73Q673c4hxBacz+dDZWUlMjIyEBkZKXc5AS+kAggAwsLCsHXrVpw5cwZut1vuctgiQkRoampCbGwsEhIS5C4nKIRcAAGATqdDdnY2Tpw4Aa/XK3c5bBEgIly/fh1DQ0PIysriTucpCskAEgQB8fHxiI+P5/4gtiBcLhcaGhqwefNmKJVKucsJGiEZQMB3IbRx40YMDQ2hs7NT7nJYCPP5fKioqEBubi40Go3c5QSVkA0g4LuLFPPz89HU1ITBwUHeE2JzbqLfZ+XKlUhMTORDr2kK6QACAI1GA6vViqqqKvj9frnLYSHmm2++wejoKNLS0uQuJShNK4BKS0uRnZ2NyMhIxMXFYffu3Whvb5/UZmxsDMXFxYiOjkZERASeeuop9Pb2TmrT2dmJXbt2Qa/XIy4uDm+88ca83dEuCAKWLl2KFStWoKmpifeC2JxxOBxoaGhAXl4eX2w4Q9Naa1VVVSguLkZ9fT3Ky8vh9XqxY8cOjIyMSG1effVVfPbZZzhy5AiqqqrQ3d2NvXv3Ssv9fj927dqF8fFx1NXV4eOPP8ZHH32EgwcPzt23+gFBELBmzRoMDw/jm2++4RBis+b1elFbW4tHH30Uer1e7nKCF81CX18fAaCqqioiInI4HKRSqejIkSNSm0uXLhEAstlsRER0/PhxUigUZLfbpTaHDh0ig8FAHo9nSp/rdDoJADmdzmnVOzo6Sn//+9/J5XJN63WM3U4URTp9+jS1tbWRKIpylxOQprqNzmq/0el0AgCWLFkCAGhubobX60VhYaHUJjU1FcnJybDZbAAAm82GdevWwWw2S22KiorgcrnQ1tZ218/xeDxwuVyTppnQ6XQoLCxEdXU1D2LGZoSI8PXXX2NgYACpqanc6TxLMw4gURRx4MAB5OfnY+3atQAAu90OtVoNk8k0qa3ZbIbdbpfa3B4+E8snlt1NaWkpjEajNCUlJc20bMTGxiIpKYn7g9iMDA0N4eLFi3j00Ue532cOzHgNFhcX48KFCzh8+PBc1nNXJSUlcDqd0tTV1TXj9xIEAWvXrkV/fz+uXbvGIcSmzOfzobq6Gvn5+Yv+cTpzZUYBtH//fhw7dgwnT55EYmKiNN9isWB8fBwOh2NS+97eXlgsFqnND8+KTfw90eaHNBoNDAbDpGk2FAoFCgsLcf78eQwPD8/qvdjiQESorq5GfHw8P05nDk0rgIgI+/fvx9GjR1FZWYlly5ZNWp6ZmQmVSoWKigppXnt7Ozo7O2G1WgEAVqsVra2t6Ovrk9qUl5fDYDAs6LUUOp0Oubm5qK2t5f4gdk9EhM7OTvh8PmzYsIH7febSdHq2X3rpJTIajXTq1Cnq6emRptHRUanNvn37KDk5mSorK6mpqYmsVitZrVZpuc/no7Vr19KOHTuopaWFysrKKDY2lkpKSua8h/1+RFGk5uZmOn36NJ/NYD9qaGiIPvnkE3K73XKXEjSmuo1OK4AA3HX68MMPpTZut5tefvllioqKIr1eT3v27KGenp5J73Pt2jXauXMn6XQ6iomJoddff528Xu+cf7mp8Pl89OWXX1JnZ+es34uFnvHxcTpy5Ah1d3fLXUpQmeo2KhAFXy+sy+WC0WiE0+mcdX8Q8N0IdmVlZdixYwf0ej3vYjMA3x16NTc3SwPd8f+LqZvqNsrnEQFotVrk5OTgxIkTPJ40k1y/fh23bt3ifp95xAGE/xs/KDExEV999RWfmmdwuVw4c+YM8vPzeXyfecQB9D1BELBhwwb09vaip6dH7nKYjPx+P2pra5GXl8cPE5xnHEC3USqV2LJlC2prayfdYMsWDyJCS0sLli5dyg8TXAAcQD+g1+ul64O4P2hxISJ0dXWhv78f69ev5/BZABxAPyAIApKTkxEdHY3W1lbuD1pE3G43zpw5w+M6LyAOoLuYGE/6+vXruHHjBofQIuD3+1FdXY1NmzYhPDxc7nIWDQ6gH6FUKlFQUICGhgZ+vliIIyJ89dVXiIuLQ3x8vNzlLCocQPdgNBqRnZ2N2tpaHk86RBERuru7cf36daSnp3O/zwLjALqPxMREREdH4+zZs3woFoLcbjcaGxuxfft2Ht9HBrzG70MQBKSnp8Nut6O7u5tDKIT4fD7U1NQgJycH4eHhvPcjAw6gKQgLC8P27du5PyiE0PfP84qOjsbSpUvlLmfR4gCaoonxg06dOsXjB4WArq4uDA8PY+PGjbznIyMOoCmaeL5YbGwsP28+iBERhoaGcPbsWRQUFPD1PjLjAJqGiWEZXC4XP28+SPn9fhw/fhx5eXn8HPcAwAE0TRPPm29pacGtW7d4TyiIiKKIhoYGrFu3DhaLhQ+9AgAH0AxoNBrk5+ejqqqK+4OCBBGho6MDPp8PqampcpfDvscBNAOCICA6Ohpr1qyBzWbjm1YDHBGhr68PTU1N/Bz3AMO/xAwJgoDly5dDqVSira2ND8UCmMfjQX19PX76059CrVbLXQ67DQfQLAiCgJycHFy5coUHMQtQfr8fNTU1yMjIQGRkpNzlsB/gAJollUqFxx57DI2NjfyQwwAzMai8yWRCYmIidzoHIA6gOaDX62G1WlFVVQWv1yt3OQzfhc///vc/uN1ufqJFAOMAmgOCICA2NhYpKSmor6/nTmmZEREGBgZw8eJFbNq0iS82DGAcQHNEEASkpaVBo9Fwp7TM3G43vvjiCxQUFCAsLEzuctg9cADNIUEQkJWVha6uLly7do1DSAY+nw/V1dXYtm0boqKi+NArwHEAzTGFQoGHH34Y586d4yulF5goiqirq0NKSgosFovc5bAp4ACaB+Hh4SgsLERtbS3GxsbkLmdRICK0trZCqVRi1apVvOcTJKYVQIcOHcL69ethMBhgMBhgtVrx+eefS8vHxsZQXFyM6OhoRERE4KmnnkJvb++k9+js7MSuXbug1+sRFxeHN954IyRvZ4iMjERmZibKysowPj4udzkhjYhw7do1DA4OIi8vj8MniEwrgBITE/HOO++gubkZTU1N2LZtG5588km0tbUBAF599VV89tlnOHLkCKqqqtDd3Y29e/dKr/f7/di1axfGx8dRV1eHjz/+GB999BEOHjw4t98qAEw87nn16tWoq6vjM2PzhIhgt9vR1tbGj1EORjRLUVFR9Je//IUcDgepVCo6cuSItOzSpUsEgGw2GxERHT9+nBQKBdntdqnNoUOHyGAwkMfjmfJnOp1OAkBOp3O25c87URTp7Nmz1NTURKIoyl1OyHG5XPTf//6XhoeH5S6F3Waq2+iM+4D8fj8OHz6MkZERWK1WNDc3w+v1orCwUGqTmpqK5ORk2Gw2AIDNZsO6detgNpulNkVFRXC5XNJeVKiZGFN6cHAQ58+f507pOTQ6Ooqamhps3ryZn+UVpKYdQK2trYiIiIBGo8G+fftw9OhRpKWlwW63Q61Ww2QyTWpvNptht9sBAHa7fVL4TCyfWPZjPB4PXC7XpCmYKBQKbN26FYODg3x6fo54PB6UlZUhPT39jv9zLHhMO4BWrVqFlpYWnDlzBi+99BKee+45XLx4cT5qk5SWlsJoNEpTUlLSvH7efAgLC0NBQQEuXboEu93OITQLfr8ftbW1yMjIQEJCAnc6B7FpB5Barcby5cuRmZmJ0tJSpKen449//CMsFgvGx8fhcDgmte/t7ZWuybBYLHecFZv4+17XbZSUlMDpdEpTV1fXdMsOCCqVCo888giampowMDAgdzlBaSJ8EhMT8cADD8hdDpulWV8HJIoiPB4PMjMzoVKpUFFRIS1rb29HZ2cnrFYrAMBqtaK1tRV9fX1Sm/LychgMBqSlpf3oZ2g0GunU/8QUrPR6PbZu3YrGxkbcunVL7nKCiiiKaGpqgk6nw8qVK3nPJxRMp2f7zTffpKqqKrp69SqdP3+e3nzzTRIEgb788ksiItq3bx8lJydTZWUlNTU1kdVqJavVKr3e5/PR2rVraceOHdTS0kJlZWUUGxtLJSUl89LDHqhEUSSHw0GHDx+mwcFBPjs2BX6/n5qamqilpYX8fr/c5bD7mOo2Oq0A+tWvfkUPPPAAqdVqio2Npe3bt0vhQ0Tkdrvp5ZdfpqioKNLr9bRnzx7q6emZ9B7Xrl2jnTt3kk6no5iYGHr99dfJ6/VOp4ygD6AJAwMDVFZWRi6XS+5SAprf76e6ujo6ffo0h0+QmOo2KhAFX2+oy+WC0WiE0+kM6sMxALh16xZqampQUFDAN0/ehSiKaG1txfj4ODIzM3k85yAx1W2Uf02ZmUwmWK1W1NXVYWhoSO5yAoooiqivr4fX6+XwCVH8i8psYjCz/Px81NXVYXBwkE/R47vwOXv2LAAgIyODwydE8a8aIEwmE3Jzc3HixIlFf4re7/ejvr4eKpWKH6MT4viXDRCCICAqKgo7d+5EY2Mjenp6FuWekNfrRVVVFTQaDdavX8/hE+L41w0wERER2Lp1K86ePYuOjo4ZhRARweFwoLu7G+Pj46DvznbOQ7Vza2RkBJWVlUhJSeGB5BcJHjA3wAiCAK1Wix07dsBms2F0dBTr1q2b9p7AoUOHcOjQIaSkpCA9PR0bN27EmjVrsGzZMhgMBmg0moDZwIkILpcL5eXlyMnJQVJSUsDUxuYXn4YPYH6/Hw0NDXC73di8eTNUKtWUX1dYWIhTp05J8wRBQHh4OEwmE1avXo1169YhIyMDa9asQUJCAqKioqSxdBZy4ycifPvttzh//jwKCgqwZMmSBftsNn+muo1yAAU4IsI333yDK1euoKCgAJGRkfcNiIGBAeTn56O9vf2e7QRBgF6vh8ViwfLly5GRkYH09HSsWbMGq1atmnLgzZQoimhra8OVK1ewa9eugNorY7PDARRCiAj9/f04deoU8vLy7nuIcu7cOeTl5c1oKNiwsDCYTCbU1NQgNTV1NmX/KCKCx+NBbW0tDAYDMjIy+PE5IYYvRAwhE9cKPfHEE2hvb0dtbe09x9GeuHhPEARERkbCaDROeQP3+XywWCzz9lQJIkJfXx+OHTuG+Ph4ZGdnc/gsYvzLB4mJzunt27ejubkZn376KbZs2YLo6OhJe0NEhIsXL0Kr1WLr1q1YvXo1lEolbty4gS+//BL9/f33/Syz2Twve5Zerxc2mw0OhwNFRUWIiIjgQ65FjgMoyCgUCmRnZ2P58uWorKxEXFwc8vLyEBYWBkEQMDo6iubmZmzduhXZ2dnSBr58+XLodDr84x//wOjoKDQaDVatWoWYmBgMDAygvb1deoTQo48+OqfBQETo6elBQ0MDEhMTefB4JuEAClImkwlPPvkkzp8/j08//RRZWVlS31BOTs4dexcTT+lYtWoVLl26hD179mDFihUQBEHq6P7kk0/g8/nuOTbTdI2OjqKxsRHd3d3YuXMnjEYj7/UwCXdCBzkiwtDQEGprayGKotT5/Ne//vWujwKqqKiAWq1GQUHBHYdudXV1aGlpQXNz86yGvSUi+Hw+fPXVV7hy5QqysrLw0EMP8V7PIjLVbZT3gIKcIAgwGAzYuXMn+vr6UF5ejrCwMKhUKng8nkltdTod3n33XdhsNjidzjveJyUlBf39/TAajTOux+PxoLW1FR0dHXjooYewd+9ePr3OfhQHUIgQBAFmsxm/+MUv0NPTA5/Ph2vXrkmn4jUaDbZt24aMjAx0dHTcEUAAkJCQgLVr1yIiImJanz2xF9ba2oquri4kJibi8ccfh16v5+Bh98QBFGKUSiUSEhLw9NNP4+bNm7h8+TJqamqQnJyM+Ph4EBE2bdqEGzduwO12S6/TarX42c9+hgcffPC+oTFx1O52u9HR0YGLFy9CoVBgxYoV2Lt3L1QqFQcPmxLuAwpxExf9dXZ2oqWlBW63G9HR0XC5XOjq6oLH40FsbCw2b96MtLS0u95zNtGn4/F4MDg4iG+//RYdHR1Qq9VISkpCamoqYmJi+M51JuErodkdiAjj4+Po6+tDR0cH7HY7Ojs7ERUVBa1WC4PBAK1WC5/PB61WC4fDAYVCgZGREYiiCK1Wi/DwcKxYsQKxsbEwmUwcOuyuuBOa3UEQBGg0GiQlJSEpKQlEBFEUIYoihoaG4Ha74fV64fF4oFarQUQwGAyIiIiQDqsmJsbmAgfQIiYIApRKJZRKJd+FzmTB+8+MMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZBOUt2JM3D/rcrlkroQxdjcT2+b97nUPygAaGBgAgFkNG8oYm39DQ0P3HGEzKANo4sbJzs7OWQ0futi4XC4kJSWhq6uLhzGZIl5nMzMxSmZ8fPw92wVlAE2MQWM0Gvk/xQwYDAZeb9PE62z6prJzwJ3QjDHZcAAxxmQTlAGk0Wjw9ttvQ6PRyF1KUOH1Nn28zuZXUI4JzRgLDUG5B8QYCw0cQIwx2XAAMcZkwwHEGJNNUAbQ+++/j5SUFGi1WuTm5qKhoUHukmRTWlqK7OxsREZGIi4uDrt370Z7e/ukNmNjYyguLkZ0dDQiIiLw1FNPobe3d1Kbzs5O7Nq1C3q9HnFxcXjjjTfg8/kW8qvI5p133oEgCDhw4IA0j9fZAqEgc/jwYVKr1fS3v/2N2tra6IUXXiCTyUS9vb1ylyaLoqIi+vDDD+nChQvU0tJCP/nJTyg5OZmGh4elNvv27aOkpCSqqKigpqYmysvLo02bNknLfT4frV27lgoLC+ncuXN0/PhxiomJoZKSEjm+0oJqaGiglJQUWr9+Pb3yyivSfF5nCyPoAignJ4eKi4ulv/1+P8XHx1NpaamMVQWOvr4+AkBVVVVERORwOEilUtGRI0ekNpcuXSIAZLPZiIjo+PHjpFAoyG63S20OHTpEBoOBPB7Pwn6BBTQ0NEQrVqyg8vJyevjhh6UA4nW2cILqEGx8fBzNzc0oLCyU5ikUChQWFsJms8lYWeBwOp0A/u+G3ebmZni93knrLDU1FcnJydI6s9lsWLduHcxms9SmqKgILpcLbW1tC1j9wiouLsauXbsmrRuA19lCCqqbUW/evAm/3z/pRwcAs9mMy5cvy1RV4BBFEQcOHEB+fj7Wrl0LALDb7VCr1TCZTJPams1m2O12qc3d1unEslB0+PBhnD17Fo2NjXcs43W2cIIqgNi9FRcX48KFC6itrZW7lIDW1dWFV155BeXl5dBqtXKXs6gF1SFYTEwMlErlHWcjent7YbFYZKoqMOzfvx/Hjh3DyZMnkZiYKM23WCwYHx+Hw+GY1P72dWaxWO66TieWhZrm5mb09fUhIyMDYWFhCAsLQ1VVFd577z2EhYXBbDbzOlsgQRVAarUamZmZqKiokOaJooiKigpYrVYZK5MPEWH//v04evQoKisrsWzZsknLMzMzoVKpJq2z9vZ2dHZ2SuvMarWitbUVfX19Upvy8nIYDAakpaUtzBdZQNu3b0draytaWlqkKSsrC88++6z0b15nC0TuXvDpOnz4MGk0Gvroo4/o4sWL9OKLL5LJZJp0NmIxeemll8hoNNKpU6eop6dHmkZHR6U2+/bto+TkZKqsrKSmpiayWq1ktVql5ROnlHfs2EEtLS1UVlZGsbGxi+qU8u1nwYh4nS2UoAsgIqI//elPlJycTGq1mnJycqi+vl7ukmQD4K7Thx9+KLVxu9308ssvU1RUFOn1etqzZw/19PRMep9r167Rzp07SafTUUxMDL3++uvk9XoX+NvI54cBxOtsYfBwHIwx2QRVHxBjLLRwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2HECMMdlwADHGZMMBxBiTDQcQY0w2/x8CdsmDl585cAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.2  -0.07], [0.6  0.07], (2,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [-0.5130131  0.       ]\n",
      "action= 1\n",
      "next_state= [-5.130925e-01 -7.937915e-05]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f6f3d24dd60>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=2048,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-400.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/5.PPO.Mountain Car')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-19.4, 9.340235543068495)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/5.PPO.Mountain Car')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhhklEQVR4nO3da2xT9/0/8PexYzvOxXbIzQTHQLk0hHvuJlwKBFLGSqHVtHV9UHVTO9owlbaq1Dwo1aRJqTbpr61bRR9Ma/tgGhPT2FYKaUMCCSHOnUAIkBYaiCFxArnYuTi+nc//QZfzwy2lSQg5vnxe0pEan6+dj084737P95zzPQIRERhjTAYKuQtgjEUuDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGxkC6APPvgAixYtQnR0NPLz89HY2ChXKYwxmcgSQP/4xz/wxhtv4N1330VrayvWrl2L4uJi9Pf3y1EOY0wmghw3o+bn5yM3Nxd//vOfAQCiKCI9PR2//vWv8fbbb891OYwxmUTN9S/0eDxoaWlBaWmp9JpCoUBRURGsVut93+N2u+F2u6WfRVHE4OAgEhMTIQjCI6+ZMTY9RISRkRGkpaVBofj+A605D6C7d+/C7/cjNTU14PXU1FRcvXr1vu8pKyvDb37zm7kojzE2i2w2G0wm0/eun/MAmonS0lK88cYb0s8OhwNmsxk2mw06nU7Gyhhj9+N0OpGeno74+PgHtpvzAEpKSoJSqURfX1/A6319fTAajfd9j0ajgUaj+c7rOp2OA4ixIPZDQyRzfhZMrVYjOzsblZWV0muiKKKyshIWi2Wuy2GMyUiWQ7A33ngDL7zwAnJycpCXl4c//OEPGBsbw4svvihHOYwxmcgSQD/96U9x584dHDp0CHa7HevWrUN5efl3BqYZY+FNluuAHpbT6YRer4fD4eAxIMYeMa/XjtHRc4iOzoBKNR8KRTwUCtUD3zPVfTQkzoIxxuTjdnehq+t5CIIGKtV8REcvh1a7BjExWdBolkKtNkOpjAUQNe3r8jiAGGM/iMgLIjfcbifc7k44HJ8CEKBQ6KBSpSImJgd6/Y8wb95PIQhTjxUOIMbYNCihUEQjKsqI6Oil0GpX/68ntBwazWIAyml9GgcQY+yB3G4BavUmJCTkQatdA612FVSqBVAq9VAo1A/12RxAjLEH6upSQqX6f1iwYB0AYVbvv+QAYow90DfnyZUQhNm/bplnRGSM3dfkHe0DAwN4VFfrcA+IsQhHRBBFER6PB3fv3sX169dx6dIlNDc34+rVq+jq6sLJkycfye/mAGIsQtzbi/F4PBgYGMDNmzdx4cIFXLhwAa2trbDZbBgYGIDH45HaRkU9upjgAGIszBERhoaG8OWXX+LKlStobW3FxYsX0dXVhf7+fni9Xoii+L3vF0URNpsNWVlZs14bBxBjEeBXv/oVysvLMTY2Nu3xHFEUUVNTgz179sz6DKQ8CM1YBFi2bBlGR0dnPJh8/vx5TExMzHJV3ANiLCJkZmZOua1Go8HKlSuxYMECOBwOtLe3o7OzE+Pj49BqtbNaFwcQY2FOEASsWbNGujsdAJRKJWJjYyGKYsBhmUqlwlNPPYXMzEzpcMtsNuOzzz7D5cuXsWnTplmtjQOIsQhgMpkQFxcHh8OBpKQk7NixAyaTCX6/H1evXsXp06fhcrmQlZWFzMzMgCdZaLVaWCyWR3ItEAcQYxEgLi4OOTk5GBwcxFNPPQWz2Sz1cHJzc6FQKHDixAmYTKb7PkZn1apVyMvLm/W6OIAYiwAqlQoLFy7E8uXLkZ6eHnA2SxAErFu3Dlu2bIHBYMDt27e/8/6YmJgHPt9rpvgsGGMRQBAEbN26FXq9/r5BolKp8POf/xzPPPMMEhMTA9YpFApYLBZERUVJV00TkbQ8DO4BMRYhduzYgfT0dFRWVsLlcgWsS05OhsFggFarxb59+/DZZ5+hq6tL6im1t7ejp6cHWq0Wo6Oj0Gq1UKlUiI6Oxrx58xAXF4cFCxYgPj4eavXUp+jgAGIsQsTGxkpXM1dUVEiPO9fr9di+fTv6+/vR0dGB4eFhEBFycnKQlpYGjUaDxMREGAyGgN6TKIqYmJjA4OAgent70dXVBYfDgdjYWCQkJEypJp6UnrEII4oienp68PXXX+POnTsYHR2F3++H2WzGkiVLMH/+/IAHgU7l6ud7Y2R4eBidnZ2wWCw8KT1jLJDX68XQ0BCuX78Oo9GIgoICpKSkICpq+pPKT7r3fQkJCVO+8JEDiLEIQETwer1ob2/HpUuXkJGRgZ/85CeIi4uTtS4OIMbCnN/vR0dHB9rb27FixQo899xzUKlUs35j6UxwADEWpogIt2/fRlNTE5KSkrB3717ExMQERfBM4gBiLAx5PB40Njair68P27Ztg8FgCKrgmcQBxFgYISL09PTg9OnTyMnJgcVigVI5vWd1zSUOIMbChN/vR1tbG/r6+rBnzx7Ex8cHZa/nXtO+FaOmpgZPPfUU0tLSIAgC/v3vfwesJyIcOnQI8+fPh1arRVFREb766quANoODg3j++eeh0+lgMBjwy1/+EqOjow/1RRiLZGNjY/j8888hCAKKi4uh0+mCPnyAGQTQ2NgY1q5diw8++OC+63/3u9/h/fffx4cffoiGhgbExsaiuLg4YDa1559/Hh0dHaioqMDx48dRU1ODl19+eebfgrEIRUSw2Ww4deoU1q9fj/Xr1wf1Idd30EMAQMeOHZN+FkWRjEYj/f73v5deGx4eJo1GQ3//+9+JiOjy5csEgJqamqQ2J0+eJEEQ6Pbt21P6vQ6HgwCQw+F4mPIZC2miKNLVq1fpn//8J42OjpIoinKXJJnqPjqrd8N3dXXBbrejqKhIek2v1yM/Px9WqxUAYLVaYTAYkJOTI7UpKiqCQqFAQ0PDfT/X7XbD6XQGLIxFMp/PB6vVijt37mDPnj2IjY0NiUOub5vVALLb7QCA1NTUgNdTU1OldXa7HSkpKQHro6KiMG/ePKnNt5WVlUGv10tLenr6bJbNWEjxer04e/YsVCoVCgsLoVKp5C5pxkJiPqDS0lI4HA5psdlscpfE2JwjIkxMTKC6uhpmsxk5OTkh2eu516wGkNFoBAD09fUFvN7X1yetMxqN6O/vD1jv8/kwODgotfk2jUYDnU4XsDAWSYgIbrcbFRUVePzxx/HYY4+FfPgAsxxAixcvhtFoRGVlpfSa0+lEQ0MDLBYLAMBisWB4eBgtLS1Sm6qqKoiiiPz8/Nksh7GwMTY2hqqqKhQUFMBkMoVF+AAzuBBxdHQU165dk37u6upCW1sb5s2bB7PZjIMHD+K3v/0tli1bhsWLF+Odd95BWloa9u7dCwBYsWIFnnzySbz00kv48MMP4fV6ceDAAfzsZz9DWlrarH0xxsIBEWF0dBSVlZXYvHkz5s2bJ3dJs2u6p9dOnz5NAL6zvPDCC0T0zanBd955h1JTU0mj0dD27dups7Mz4DMGBgboueeeo7i4ONLpdPTiiy/SyMjIlGvg0/AsUjidTjp58iQNDQ0F1Wn2HzLVfZRnRGQsCBERhoeHce7cOWzcuBEGg0HukqZlqvtoSJwFYyySEBHGx8dx7tw5WCwW6PV6uUt6ZPhmVMaCzMTEBM6cOYMNGzaE35jPt3APiLEg4na7UVlZiZycnCk/WSKUcQAxFiQmr3Bet24dUlJSwuZU+4NwADEWBLxeLyorK/HYY49hwYIFERE+AAcQY7IjIjQ2NsJoNGLx4sUREz4ABxBjsiIitLe3Q61WY+3atREVPgAHEGOyISJcunQJPT09yM7OjrjwATiAGJNNb28venp6sG3btoBnrkeSyPzWjMls8obszZs3Q61Wy12ObDiAGJtjLpcLJ06cgMViQXR0tNzlyCqkr4QOwdvYWITz+/2oqanBli1bkJSUJHc5sgvpHpDT6eQQYiFDFEVYrVaYTCaeeuZ/QjqA6urq4PF45C6DsR9ERLhy5Qo8Hg8yMzMj8ozX/YR0AGVkZKCyshI+n0/uUhj7XkSEvr4+2Gw2bNq0icPnHiEdQIsWLUJSUhLa2tr4UIwFrdHRUVitVmzevDmkn2DxKIR0AAmCgOzsbDidTty8eZNDiAUdr9eLM2fOIDc3F1qtVu5ygk5IBxAAKJVKbNy4Ec3NzRgaGpK7HMYkoiiivr4ey5cvj6gbTKcj5AMIAFQqFZ544gmcPXsWbrdb7nIYk26zUCgUWLZsGYfP9wiLABIEAYmJiVi9ejXq6uogiqLcJbEIRkS4c+cOurq6kJ+fH7G3WUxF2GwZQRCwePFixMbG4vz58zwexGTjcrlQX1+Pbdu2ISoqpK/1feTCJoCAb0IoKysLdrsddrudQ4jNOZ/Ph6qqKmRlZSE+Pl7ucoJeWAUQAERFRWHr1q1oaGiAy+WSuxwWQYgIzc3NSE5OxoIFC+QuJySEXQABgFarRW5uLk6dOgWv1yt3OSwCEBFu3bqFkZER5OTk8KDzFIVlAAmCgLS0NKSlpfF4EJsTTqcTjY2N2LRpE5RKpdzlhIywDCDgmxBav349RkZG0N3dLXc5LIz5fD5UVlYiPz8fGo1G7nJCStgGEPDNRYqFhYVobm7G4OAg94TYrJsc91m+fDlMJhMfek1TWAcQAGg0GlgsFlRXV8Pv98tdDgsz169fx/j4ODIzM+UuJSRNK4DKysqQm5uL+Ph4pKSkYO/evejs7AxoMzExgZKSEiQmJiIuLg7PPvss+vr6Atp0d3dj9+7diImJQUpKCt56661Hdke7IAiYP38+li1bhubmZu4FsVkzPDyMxsZGFBQU8MWGMzStrVZdXY2SkhLU19ejoqICXq8XO3fuxNjYmNTm9ddfx6effoqjR4+iuroaPT09eOaZZ6T1fr8fu3fvhsfjQV1dHT755BN8/PHHOHTo0Ox9q28RBAErV67E6Ogorl+/ziHEHprX60VtbS127NiBmJgYucsJXfQQ+vv7CQBVV1cTEdHw8DCpVCo6evSo1ObKlSsEgKxWKxERnThxghQKBdntdqnN4cOHSafTkdvtntLvdTgcBIAcDse06h0fH6e//e1v5HQ6p/U+xu4liiKdO3eOOjo6SBRFucsJSlPdRx+q3+hwOAAA8+bNAwC0tLTA6/WiqKhIapORkQGz2Qyr1QoAsFqtWL16NVJTU6U2xcXFcDqd6OjouO/vcbvdcDqdActMaLVaFBUVoaamhicxYzNCRPjqq68wMDCAjIwMHnR+SDMOIFEUcfDgQRQWFmLVqlUAALvdDrVaDYPBENA2NTUVdrtdanNv+Eyun1x3P2VlZdDr9dKSnp4+07KRnJyM9PR0Hg9iMzIyMoLLly9jx44dPO4zC2a8BUtKSnDp0iUcOXJkNuu5r9LSUjgcDmmx2Wwz/ixBELBq1SrcuXMHN27c4BBiU+bz+VBTU4PCwsKIf5zObJlRAB04cADHjx/H6dOnYTKZpNeNRiM8Hg+Gh4cD2vf19cFoNEptvn1WbPLnyTbfptFooNPpApaHoVAoUFRUhIsXL2J0dPShPotFBiJCTU0N0tLS+HE6s2haAUREOHDgAI4dO4aqqiosXrw4YH12djZUKhUqKyul1zo7O9Hd3Q2LxQIAsFgsaG9vR39/v9SmoqICOp1uTq+l0Gq1yM/PR21tLY8HsQciInR3d8Pn82HdunU87jObpjOy/corr5Ber6czZ85Qb2+vtIyPj0tt9u/fT2azmaqqqqi5uZksFgtZLBZpvc/no1WrVtHOnTupra2NysvLKTk5mUpLS2d9hP2HiKJILS0tdO7cOT6bwb7XyMgI/etf/yKXyyV3KSFjqvvotAIIwH2Xjz76SGrjcrno1VdfpYSEBIqJiaF9+/ZRb29vwOfcuHGDdu3aRVqtlpKSkujNN98kr9c7619uKnw+H33xxRfU3d390J/Fwo/H46GjR49ST0+P3KWElKnuowJR6I3COp1O6PV6OByOhx4PAr6Zwa68vBw7d+5ETEwMd7EZgG8OvVpaWqSJ7vjfxdRNdR/l84gAoqOjkZeXh1OnTvF80kxy69YtDA0N8bjPI8QBhP+bP8hkMuHChQt8ap7B6XSioaEBhYWFPL/PI8QB9D+CIGDdunXo6+tDb2+v3OUwGfn9ftTW1qKgoIAfJviIcQDdQ6lUYvPmzaitrQ24wZZFDiJCW1sb5s+fzw8TnAMcQN8SExMjXR/E40GRhYhgs9lw584drFmzhsNnDnAAfYsgCDCbzUhMTER7ezuPB0UQl8uFhoYGntd5DnEA3cfkfNK3bt3C7du3OYQigN/vR01NDTZs2IDY2Fi5y4kYHEDfQ6lUYuPGjWhsbOTni4U5IsKFCxeQkpKCtLQ0ucuJKBxAD6DX65Gbm4va2lqeTzpMERF6enpw69YtrF27lsd95hgH0A8wmUxITExEa2srH4qFIZfLhaamJmzfvp3n95EBb/EfIAgC1q5dC7vdjp6eHg6hMOLz+XD27Fnk5eUhNjaWez8y4ACagqioKGzfvp3Hg8II/e95XomJiZg/f77c5UQsDqApmpw/6MyZMzx/UBiw2WwYHR3F+vXruecjIw6gKZp8vlhycjI/bz6EERFGRkbQ2tqKjRs38vU+MuMAmobJaRmcTic/bz5E+f1+nDhxAgUFBfwc9yDAATRNk8+bb2trw9DQEPeEQogoimhsbMTq1athNBr50CsIcADNgEajQWFhIaqrq3k8KEQQEa5duwafz4eMjAy5y2H/wwE0A4IgIDExEStXroTVauWbVoMcEaG/vx/Nzc38HPcgw3+JGRIEAUuXLoVSqURHRwcfigUxt9uN+vp6/PjHP4ZarZa7HHYPDqCHIAgC8vLy8OWXX/IkZkHK7/fj7NmzyMrKQnx8vNzlsG/hAHpIKpUKTz75JJqamvghh0FmclJ5g8EAk8nEg85BiANoFsTExMBisaC6uhper1fuchi+CZ+vv/4aLpeLn2gRxDiAZoEgCEhOTsaiRYtQX1/Pg9IyIyIMDAzg8uXL2LBhA19sGMQ4gGaJIAjIzMyERqPhQWmZuVwufP7559i4cSOioqLkLoc9AAfQLBIEATk5ObDZbLhx4waHkAx8Ph9qamqwbds2JCQk8KFXkOMAmmUKhQJbtmzB+fPn+UrpOSaKIurq6rBo0SIYjUa5y2FTwAH0CMTGxqKoqAi1tbWYmJiQu5yIQERob2+HUqnE448/zj2fEDGtADp8+DDWrFkDnU4HnU4Hi8WCkydPSusnJiZQUlKCxMRExMXF4dlnn0VfX1/AZ3R3d2P37t2IiYlBSkoK3nrrrbC8nSE+Ph7Z2dkoLy+Hx+ORu5ywRkS4ceMGBgcHUVBQwOETQqYVQCaTCe+99x5aWlrQ3NyMbdu24emnn0ZHRwcA4PXXX8enn36Ko0ePorq6Gj09PXjmmWek9/v9fuzevRsejwd1dXX45JNP8PHHH+PQoUOz+62CwOTjnlesWIG6ujo+M/aIEBHsdjs6Ojr4McqhiB5SQkIC/eUvf6Hh4WFSqVR09OhRad2VK1cIAFmtViIiOnHiBCkUCrLb7VKbw4cPk06nI7fbPeXf6XA4CAA5HI6HLf+RE0WRWltbqbm5mURRlLucsON0Ouk///kPjY6Oyl0Ku8dU99EZjwH5/X4cOXIEY2NjsFgsaGlpgdfrRVFRkdQmIyMDZrMZVqsVAGC1WrF69WqkpqZKbYqLi+F0OqVeVLiZnFN6cHAQFy9e5EHpWTQ+Po6zZ89i06ZN/CyvEDXtAGpvb0dcXBw0Gg3279+PY8eOITMzE3a7HWq1GgaDIaB9amoq7HY7AMButweEz+T6yXXfx+12w+l0BiyhRKFQYOvWrRgcHOTT87PE7XajvLwca9eu/c6/ORY6ph1Ajz/+ONra2tDQ0IBXXnkFL7zwAi5fvvwoapOUlZVBr9dLS3p6+iP9fY9CVFQUNm7ciCtXrsBut3MIPQS/34/a2lpkZWVhwYIFPOgcwqYdQGq1GkuXLkV2djbKysqwdu1a/PGPf4TRaITH48Hw8HBA+76+PumaDKPR+J2zYpM/P+i6jdLSUjgcDmmx2WzTLTsoqFQqPPHEE2hubsbAwIDc5YSkyfAxmUxYuHCh3OWwh/TQ1wGJogi3243s7GyoVCpUVlZK6zo7O9Hd3Q2LxQIAsFgsaG9vR39/v9SmoqICOp0OmZmZ3/s7NBqNdOp/cglVMTEx2Lp1K5qamjA0NCR3OSFFFEU0NzdDq9Vi+fLl3PMJB9MZ2X777bepurqaurq66OLFi/T222+TIAj0xRdfEBHR/v37yWw2U1VVFTU3N5PFYiGLxSK93+fz0apVq2jnzp3U1tZG5eXllJycTKWlpY9khD1YiaJIw8PDdOTIERocHOSzY1Pg9/upubmZ2trayO/3y10O+wFT3UenFUC/+MUvaOHChaRWqyk5OZm2b98uhQ8RkcvloldffZUSEhIoJiaG9u3bR729vQGfcePGDdq1axdptVpKSkqiN998k7xe73TKCPkAmjQwMEDl5eXkdDrlLiWo+f1+qquro3PnznH4hIip7qMCUeiNhjqdTuj1ejgcjpA+HAOAoaEhnD17Fhs3buSbJ+9DFEW0t7fD4/EgOzub53MOEVPdR/mvKTODwQCLxYK6ujqMjIzIXU5QEUUR9fX18Hq9HD5hiv+iMpuczKywsBB1dXUYHBzkU/T4JnxaW1sBAFlZWRw+YYr/qkHCYDAgPz8fp06divhT9H6/H/X19VCpVPwYnTDHf9kgIQgCEhISsGvXLjQ1NaG3tzcie0JerxfV1dXQaDRYs2YNh0+Y479ukImLi8PWrVvR2tqKa9euRVQIjY2NoaqqCosWLeKJ5CMEB1CQEQQB0dHR2LlzJ3p7e3Hx4sWwn8qDiOBwOHDy5EmsWLECixcv5vCJEBxAQUqlUqGwsBDj4+M4c+ZM2D7uh4hw8+ZNVFdXY9u2bTCbzRw+EYQDKIgplUoUFBTAbDajoqICTqczrA7JRFHEpUuX0NLSgp07dyIhIUHuktgc4wAKcoIgYMmSJcjJyUF5eTlsNlvIhxARYWJiAlVVVXC5XHj66acRHR3NPZ8IxAEUAiavFdqzZw86OztRW1sbsvNoExH6+/tx/PhxpKWlITc3l5/dFcE4gELE5OD09u3bER0djf/+97+4e/duSPWGvF4vzp49i4aGBhQXF2PFihXc64lw/L+eEKNQKJCbm4ulS5eiqqoKKSkpKCgoQFRUVNDuzESE3t5eNDY2wmQy8eTxTMI3o4YoIoLf78fFixfR1dWFnJwcpKenB92Fe+Pj42hqakJPTw927doFvV4ftEHJZs9U91EOoBBHRBgZGUFtbS1EUURBQQESExNl3cmJCD6fDxcuXMCXX36JnJwcLFmyhHs9EYQDKMJMDu5WVVUhPj4e+fn5SEpKmvMgcrvdaG9vx7Vr17BkyRKsXr0aGo2Gez0RhgMoQvn9fmm8xefzIS8vDwsWLHikY0STvbD29nbYbDaYTCasX78eMTExHDwRigMogtE3M13izp07Um9kxYoVWL58OVJSUqBQKB4qGCb/ybhcLly7dg2XL1+GQqHAsmXLsHLlSqhUKg6eCMcBxAB8ExZutxvd3d1oa2uDy+VCUlIS0tLSsHDhQmi1WukQ6ftCY3JMx+12Y3BwEDdv3sS1a9egVquRnp6OjIwMJCUlBd0AOJMPBxD7DiKCx+NBf38/rl27hqGhIdjtdsTFxUEQBMTHxyM6Oho+nw/R0dEYHh6GQqHA2NgYRFFEdHQ0YmNjsWzZMiQnJ8NgMHDosPua6j7K1wFFEEEQoNFokJ6ejvT0dBARRFGEKIoYGRmBy+WC1+uF2+2GWq0GEUGn0yEuLk46rHpQT4mx6eIAimCCIECpVEKpVGLevHlyl8MiEPefGWOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMmGA4gxJhsOIMaYbDiAGGOy4QBijMkmJG/FmLx/1ul0ylwJY+x+JvfNH7rXPSQDaGBgAACQnp4ucyWMsQcZGRmBXq//3vUhGUCTN052d3c/8MuxQE6nE+np6bDZbDyNyRTxNpuZyVky09LSHtguJANocg4avV7P/yhmQKfT8XabJt5m0zeVzgEPQjPGZMMBxBiTTUgGkEajwbvvvguNRiN3KSGFt9v08TZ7tEJyTmjGWHgIyR4QYyw8cAAxxmTDAcQYkw0HEGNMNiEZQB988AEWLVqE6Oho5Ofno7GxUe6SZFNWVobc3FzEx8cjJSUFe/fuRWdnZ0CbiYkJlJSUIDExEXFxcXj22WfR19cX0Ka7uxu7d+9GTEwMUlJS8NZbb8Hn883lV5HNe++9B0EQcPDgQek13mZzhELMkSNHSK1W01//+lfq6Oigl156iQwGA/X19cldmiyKi4vpo48+okuXLlFbWxv96Ec/IrPZTKOjo1Kb/fv3U3p6OlVWVlJzczMVFBTQhg0bpPU+n49WrVpFRUVFdP78eTpx4gQlJSVRaWmpHF9pTjU2NtKiRYtozZo19Nprr0mv8zabGyEXQHl5eVRSUiL97Pf7KS0tjcrKymSsKnj09/cTAKquriYiouHhYVKpVHT06FGpzZUrVwgAWa1WIiI6ceIEKRQKstvtUpvDhw+TTqcjt9s9t19gDo2MjNCyZcuooqKCtmzZIgUQb7O5E1KHYB6PBy0tLSgqKpJeUygUKCoqgtVqlbGy4OFwOAD83w27LS0t8Hq9AdssIyMDZrNZ2mZWqxWrV69Gamqq1Ka4uBhOpxMdHR1zWP3cKikpwe7duwO2DcDbbC6F1M2od+/ehd/vD/ijA0BqaiquXr0qU1XBQxRFHDx4EIWFhVi1ahUAwG63Q61Ww2AwBLRNTU2F3W6X2txvm06uC0dHjhxBa2srmpqavrOOt9ncCakAYg9WUlKCS5cuoba2Vu5SgprNZsNrr72GiooKREdHy11ORAupQ7CkpCQolcrvnI3o6+uD0WiUqargcODAARw/fhynT5+GyWSSXjcajfB4PBgeHg5of+82MxqN992mk+vCTUtLC/r7+5GVlYWoqChERUWhuroa77//PqKiopCamsrbbI6EVACp1WpkZ2ejsrJSek0URVRWVsJischYmXyICAcOHMCxY8dQVVWFxYsXB6zPzs6GSqUK2GadnZ3o7u6WtpnFYkF7ezv6+/ulNhUVFdDpdMjMzJybLzKHtm/fjvb2drS1tUlLTk4Onn/+eem/eZvNEblHwafryJEjpNFo6OOPP6bLly/Tyy+/TAaDIeBsRCR55ZVXSK/X05kzZ6i3t1daxsfHpTb79+8ns9lMVVVV1NzcTBaLhSwWi7R+8pTyzp07qa2tjcrLyyk5OTmiTinfexaMiLfZXAm5ACIi+tOf/kRms5nUajXl5eVRfX293CXJBsB9l48++khq43K56NVXX6WEhASKiYmhffv2UW9vb8Dn3Lhxg3bt2kVarZaSkpLozTffJK/XO8ffRj7fDiDeZnODp+NgjMkmpMaAGGPhhQOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsuEAYozJhgOIMSYbDiDGmGw4gBhjsvn/lQ8adFtehMQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-6.0 107 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 100.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        action = action.item()\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
